From 95d3856e97242a79e735138e71d6c8ab54b2beef Mon Sep 17 00:00:00 2001 From: "john.levon@sun.com" Date: Tue, 23 Jan 2007 19:06:31 -0800 Subject: [PATCH] Close save-after-restore race. Make xc_linux_save() wait for the frame_list_list MFN to be updated by the domain before trying to use it. Make Linux set the top-level MFN /after/ updating the other MFN lists. Signed-off-by: John Levon --- .../arch/i386/kernel/setup-xen.c | 4 +- .../arch/x86_64/kernel/setup-xen.c | 4 +- .../drivers/xen/core/machine_reboot.c | 5 +-- tools/libxc/xc_linux_save.c | 44 ++++++++++++++++--- 4 files changed, 43 insertions(+), 14 deletions(-) diff --git a/linux-2.6-xen-sparse/arch/i386/kernel/setup-xen.c b/linux-2.6-xen-sparse/arch/i386/kernel/setup-xen.c index 946ecffb8d..4d0d6e0558 100644 --- a/linux-2.6-xen-sparse/arch/i386/kernel/setup-xen.c +++ b/linux-2.6-xen-sparse/arch/i386/kernel/setup-xen.c @@ -1736,8 +1736,6 @@ void __init setup_arch(char **cmdline_p) * frames that make up the p2m table. Used by save/restore */ pfn_to_mfn_frame_list_list = alloc_bootmem_low_pages(PAGE_SIZE); - HYPERVISOR_shared_info->arch.pfn_to_mfn_frame_list_list = - virt_to_mfn(pfn_to_mfn_frame_list_list); fpp = PAGE_SIZE/sizeof(unsigned long); for (i=0, j=0, k=-1; i< max_pfn; i+=fpp, j++) { @@ -1754,6 +1752,8 @@ void __init setup_arch(char **cmdline_p) virt_to_mfn(&phys_to_machine_mapping[i]); } HYPERVISOR_shared_info->arch.max_pfn = max_pfn; + HYPERVISOR_shared_info->arch.pfn_to_mfn_frame_list_list = + virt_to_mfn(pfn_to_mfn_frame_list_list); } /* diff --git a/linux-2.6-xen-sparse/arch/x86_64/kernel/setup-xen.c b/linux-2.6-xen-sparse/arch/x86_64/kernel/setup-xen.c index 8b3b67ad95..f49110cf90 100644 --- a/linux-2.6-xen-sparse/arch/x86_64/kernel/setup-xen.c +++ b/linux-2.6-xen-sparse/arch/x86_64/kernel/setup-xen.c @@ -817,8 +817,6 @@ void __init setup_arch(char **cmdline_p) * save/restore. */ pfn_to_mfn_frame_list_list = alloc_bootmem_pages(PAGE_SIZE); - HYPERVISOR_shared_info->arch.pfn_to_mfn_frame_list_list = - virt_to_mfn(pfn_to_mfn_frame_list_list); fpp = PAGE_SIZE/sizeof(unsigned long); for (i=0, j=0, k=-1; i< end_pfn; i+=fpp, j++) { @@ -835,6 +833,8 @@ void __init setup_arch(char **cmdline_p) virt_to_mfn(&phys_to_machine_mapping[i]); } HYPERVISOR_shared_info->arch.max_pfn = end_pfn; + HYPERVISOR_shared_info->arch.pfn_to_mfn_frame_list_list = + virt_to_mfn(pfn_to_mfn_frame_list_list); } } diff --git a/linux-2.6-xen-sparse/drivers/xen/core/machine_reboot.c b/linux-2.6-xen-sparse/drivers/xen/core/machine_reboot.c index b69efd61f9..07de13c835 100644 --- a/linux-2.6-xen-sparse/drivers/xen/core/machine_reboot.c +++ b/linux-2.6-xen-sparse/drivers/xen/core/machine_reboot.c @@ -105,9 +105,6 @@ static void post_suspend(int suspend_cancelled) memset(empty_zero_page, 0, PAGE_SIZE); - HYPERVISOR_shared_info->arch.pfn_to_mfn_frame_list_list = - virt_to_mfn(pfn_to_mfn_frame_list_list); - fpp = PAGE_SIZE/sizeof(unsigned long); for (i = 0, j = 0, k = -1; i < max_pfn; i += fpp, j++) { if ((j % fpp) == 0) { @@ -120,6 +117,8 @@ static void post_suspend(int suspend_cancelled) virt_to_mfn(&phys_to_machine_mapping[i]); } HYPERVISOR_shared_info->arch.max_pfn = max_pfn; + HYPERVISOR_shared_info->arch.pfn_to_mfn_frame_list_list = + virt_to_mfn(pfn_to_mfn_frame_list_list); } #else /* !(defined(__i386__) || defined(__x86_64__)) */ diff --git a/tools/libxc/xc_linux_save.c b/tools/libxc/xc_linux_save.c index ef64d1ef50..047bbf7e95 100644 --- a/tools/libxc/xc_linux_save.c +++ b/tools/libxc/xc_linux_save.c @@ -404,6 +404,33 @@ static int suspend_and_state(int (*suspend)(int), int xc_handle, int io_fd, return -1; } +/* +** Map the top-level page of MFNs from the guest. The guest might not have +** finished resuming from a previous restore operation, so we wait a while for +** it to update the MFN to a reasonable value. +*/ +static void *map_frame_list_list(int xc_handle, uint32_t dom, + shared_info_t *shinfo) +{ + int count = 100; + void *p; + + while (count-- && shinfo->arch.pfn_to_mfn_frame_list_list == 0) + usleep(10000); + + if (shinfo->arch.pfn_to_mfn_frame_list_list == 0) { + ERROR("Timed out waiting for frame list updated."); + return NULL; + } + + p = xc_map_foreign_range(xc_handle, dom, PAGE_SIZE, PROT_READ, + shinfo->arch.pfn_to_mfn_frame_list_list); + + if (p == NULL) + ERROR("Couldn't map p2m_frame_list_list (errno %d)", errno); + + return p; +} /* ** During transfer (or in the state file), all page-table pages must be @@ -669,14 +696,11 @@ int xc_linux_save(int xc_handle, int io_fd, uint32_t dom, uint32_t max_iters, max_pfn = live_shinfo->arch.max_pfn; - live_p2m_frame_list_list = - xc_map_foreign_range(xc_handle, dom, PAGE_SIZE, PROT_READ, - live_shinfo->arch.pfn_to_mfn_frame_list_list); + live_p2m_frame_list_list = map_frame_list_list(xc_handle, dom, + live_shinfo); - if (!live_p2m_frame_list_list) { - ERROR("Couldn't map p2m_frame_list_list (errno %d)", errno); + if (!live_p2m_frame_list_list) goto out; - } live_p2m_frame_list = xc_map_foreign_batch(xc_handle, dom, PROT_READ, @@ -1175,8 +1199,14 @@ int xc_linux_save(int xc_handle, int io_fd, uint32_t dom, uint32_t max_iters, ctxt.ctrlreg[3] = xen_pfn_to_cr3(mfn_to_pfn(xen_cr3_to_pfn(ctxt.ctrlreg[3]))); + /* + * Reset the MFN to be a known-invalid value. See map_frame_list_list(). + */ + memcpy(page, live_shinfo, PAGE_SIZE); + ((shared_info_t *)page)->arch.pfn_to_mfn_frame_list_list = 0; + if (!write_exact(io_fd, &ctxt, sizeof(ctxt)) || - !write_exact(io_fd, live_shinfo, PAGE_SIZE)) { + !write_exact(io_fd, page, PAGE_SIZE)) { ERROR("Error when writing to state file (1) (errno %d)", errno); goto out; } -- 2.30.2